﻿using demo86.Models;
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.Web.Mvc;

namespace demo86.Controllers
{
    public class TestController : Controller
    {
        ResultDbContext db = new ResultDbContext();//数据库上下文
        static WebClient wc = new WebClient();
        static HtmlWeb htmlWeb = new HtmlWeb();
        static HtmlDocument htmlDoc = new HtmlDocument();
        //static string SearchText;//网页搜索文本框
        static List<Result> listResult = new List<Result>();//存抓取的最终结果
        #region 高校网站地址一个
        Dictionary<string, string> InitalUrl = new Dictionary<string, string> {//存高校所要抓取的网页
            {"福建师范大学","https://www.fjnu.edu.cn/13/list.htm" },

            { "福建江夏学院","http://www.fjjxu.edu.cn/html/xwzx/xxxw/1.html" },

            { "福建福州大学志诚学院","http://www.fdzcxy.com/list.asp?BCid=2&SCid=7" },

            { "福建警察学院","http://www.fjpsc.edu.cn/3dc229a0-9481-42ed-b6ff-32da44390138_0.html" },

            { "福建华南女子学院","http://www.hnwomen.com.cn/xin/news" },

            { "福建闽江学院","http://www.mju.edu.cn/html/xxyw/1.html" },

            { "福建农林大学","http://www.fafu.edu.cn/5280/list.htm" },

            { "福建福州大学","http://news.fzu.edu.cn/html/fdyw/1.html"},

            { "福建师范大学协和学院","http://cuc.fjnu.edu.cn/channels/4351.html" },
        };
        #endregion
        #region 高校网站地址多个
        Dictionary<string, string[]> InitalUrl2 = new Dictionary<string, string[]> {//存高校所要抓取的网页
            {"福建师范大学",new string[] { "https://www.fjnu.edu.cn/13/list.htm", "https://www.fjnu.edu.cn/19/list.htm",
                "https://www.fjnu.edu.cn/18/list.htm", "https://www.fjnu.edu.cn/90/list.htm", "https://www.fjnu.edu.cn/101/list.htm", "https://www.fjnu.edu.cn/98/list.htm" } },

            { "福建江夏学院",new string[] { "http://www.fjjxu.edu.cn/html/xwzx/xxxw/1.html" }  },

            { "福建福州大学志诚学院",new string[] { "http://www.fdzcxy.com/list.asp?BCid=2&SCid=7" }  },

            { "福建警察学院",new  string[] { "http://www.fjpsc.edu.cn/3dc229a0-9481-42ed-b6ff-32da44390138_0.html" }  },

            { "福建华南女子学院",new string[] { "http://www.hnwomen.com.cn/xin/news" }  },

            { "福建闽江学院",new string[] { "http://www.mju.edu.cn/html/xxyw/1.html" } },

            { "福建农林大学",new string[] {  "http://www.fafu.edu.cn/5280/list.htm"} },

            { "福建福州大学",new string[] {  "http://news.fzu.edu.cn/html/fdyw/1.html"}},

            { "福建师范大学协和学院",new string[] { "http://cuc.fjnu.edu.cn/channels/4351.html" }  },//, "http://cuc.fjnu.edu.cn/channels/4352.html"
        };
        #endregion

        #region 高校网站xPath
        Dictionary<string, string[]> DicXpath = new Dictionary<string, string[]> {//各种Xpath
            {"福建师范大学",new string[] { "//div[@id='wp_news_w4']/table/*/tr[@class='firstRow']", ".//td[@align='left']/a", ".//td[@align='left']/a" , ".//div[@style='white-space:nowrap']", "//div[@class='Article_Content']" } },

            { "福建江夏学院",new string[] { "//div[@class='list_main_content']/ul/li", ".//span[@class='ecms_list_title']/a", ".//span[@class='ecms_list_title']/a",".//span[@class='ecms_list_time']","//div[@class='detail_content_display']/div"} },

            { "福建福州大学志诚学院",new string[] { "//div[@class='mainlist']/ul/li", ".//a", ".//a", ".//span",/* "//*[@id='content_show']/div[2]"*/ "//div[@class='contentBody']/p" } },

            { "福建警察学院",new string[] { "//table[@class='table1']", ".//td/a", ".//td/a", ".//td[2]", "//div[@class='newslist']" } },

            { "福建华南女子学院",new string[] { "//ul[@class='news']/li", ".//a[@id='n1']", ".//a[@id='n1']", ".//a[@id='n3']" , "//div[@align='center']/div[1]" } },

            { "福建闽江学院",new string[] { "//div[@class='list_main_content']/ul/li", ".//span[@class='list_title']/a", ".//span[@class='list_title']/a", ".//span[@class='list_time']", "//div[@class='detail_content_display']/div[1]" } },

            { "福建农林大学",new string[] { "//div[@id='wp_news_w3']/table/*/tr", ".//a", ".//td[@align='left']/a", ".//div[@style='white-space:nowrap']", "//div[@class='Article_Content']" } },

            { "福建福州大学",new string[] { "//div[@class='list_main_content']/*/li", ".//a", ".//a", ".//span","//div[@id='news_content_display']"}},

            { "福建师范大学协和学院",new string[] { "//div[@class='list-normal']/h2", ".//a", ".//a", ".//span","//table[@style='MARGIN-LEFT: 15px']"}}

        };
        #endregion
        static string innerHtml;//网页的源代码
        static string[] test = new string[50];

        public object RootNode { get; private set; }

        // GET: Test
        [HttpGet]
        public ActionResult Index()
        {
            //List<string> a = new List<string>(InitalUrl.Values);
            //for (int i = 0; i < a.Count; i++)
            //{
            //    test[i] = SplitUrl(a[i].ToString());
            //}
            return View();
        }
        [HttpPost]
        public ActionResult GetHtml(string SearchText)
        {
            listResult.Clear();
            // Get_Html(listResult, InitalUrl, DicXpath, SearchText);
            Get_Html2(listResult, InitalUrl2, DicXpath, SearchText);
            return View("show", listResult);
        }
        /// <summary>
        /// 获取根网站
        /// </summary>
        /// <param name="str"></param>
        /// <returns></returns>
        public string SplitUrl(string str)
        {

            string NetAddress1 = str.Remove(str.IndexOf('/', 8));
            //string[] NetAddress = Regex.Split(str, @"/", RegexOptions.IgnoreCase);
            // return NetAddress[0]+"//"+NetAddress[1]+NetAddress[2];
            return NetAddress1;
        }
        /// <summary>
        /// 判断网页是否乱码
        /// </summary>
        /// <param name="txt"></param>
        /// <returns></returns>
        public bool isLuan(string txt)
        {
            var bytes = Encoding.UTF8.GetBytes(txt);
            //239 191 189
            for (var i = 0; i < bytes.Length; i++)
            {
                if (i < bytes.Length - 3)
                    if (bytes[i] == 239 && bytes[i + 1] == 191 && bytes[i + 2] == 189)
                    {
                        return true;
                    }
            }
            return false;
        }
        public string DateSolve(string str)
        {
            string date = Regex.Match(str, @"[1-2][0-9][0-9][0-9]\/[0-1]{0,1}[0-9]\/[0-3]{0,1}[0-9]").Groups[0].Value;
            if (date != "")
            {
                return date;
            }
            else
            {
                return Regex.Match(str, @"[1-2][0-9][0-9][0-9]-[0-1]{0,1}[0-9]-[0-3]{0,1}[0-9]").Groups[0].Value;
            }

        }
      /// <summary>
      /// /一个高校一个地址
      /// </summary>
      /// <param name="list"></param>
      /// <param name="link"></param>
      /// <param name="xpath"></param>
      /// <param name="searchText"></param>
        public void Get_Html(List<Result> list, Dictionary<string, string> link, Dictionary<string, string[]> xpath, string searchText)
        {

            wc.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1)");
            string encstr;
            innerHtml = Encoding.GetEncoding("utf-8").GetString(wc.DownloadData(link[searchText]));
            // innerHtml = Encoding.GetEncoding("utf-8").GetString(htmlWeb.Load(link[searchText]).));
            if (!isLuan(innerHtml)) { encstr = "utf-8"; } else { encstr = "gb2312"; }
            htmlWeb.OverrideEncoding = Encoding.GetEncoding(encstr); ;
            htmlDoc = htmlWeb.Load(link[searchText]);
            HtmlNode rootNode = htmlDoc.DocumentNode;
            //innerHtml = rootNode.InnerHtml;
            //ViewBag.Html = innerHtml;
            ViewBag.Html = "";
            rootNode.SelectNodes(xpath[searchText][0]).AsParallel().ToList().ForEach(
                html =>
                {
                    string href = html.SelectSingleNode(xpath[searchText][1]).Attributes["href"].Value;
                    list.Add(new Result
                    {
                        url = href.Contains("http") == false ? SplitUrl(link[searchText]) + href : href,
                        title = html.SelectSingleNode(xpath[searchText][2]).InnerText,
                        //date = DateTime.Parse(html.SelectSingleNode(xpath[searchText][3]).InnerText),
                        date = DateTime.Parse(DateSolve(html.SelectSingleNode(xpath[searchText][3]).InnerText.Replace('.', '-'))),
                        Where = searchText,
                        Type = searchText + "新闻"
                    });
                }
                );
            ViewBag.Count = list.Count();
        }
        /// <summary>
        /// 一个高校多个网址解析
        /// </summary>
        /// <param name="list"></param>
        /// <param name="link"></param>
        /// <param name="xpath"></param>
        /// <param name="searchText"></param>
        public void Get_Html2(List<Result> list, Dictionary<string, string[]> link, Dictionary<string, string[]> xpath, string searchText)
        {
            for (int i = 0; i < link[searchText].Count(); i++)
            {
                wc.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1)");
                string encstr;
                innerHtml = Encoding.GetEncoding("utf-8").GetString(wc.DownloadData(link[searchText][i]));
                // innerHtml = Encoding.GetEncoding("utf-8").GetString(htmlWeb.Load(link[searchText]).));
                if (!isLuan(innerHtml)) { encstr = "utf-8"; } else { encstr = "gb2312"; }
                htmlWeb.OverrideEncoding = Encoding.GetEncoding(encstr); ;
                htmlDoc = htmlWeb.Load(link[searchText][i]);
                HtmlNode rootNode = htmlDoc.DocumentNode;
                //innerHtml = rootNode.InnerHtml;
                //ViewBag.Html = innerHtml;
                ViewBag.Html = "";
                rootNode.SelectNodes(xpath[searchText][0]).AsParallel().ToList().ForEach(
                    html =>
                    {
                        try
                        {
                            string href = html.SelectSingleNode(xpath[searchText][1]).Attributes["href"].Value;
                            if (href.Substring(0, 1) != "/")
                            {
                                href = "/" + href;
                            }
                            string loaourl = href.Contains("http") == false ? SplitUrl(link[searchText][i]) + href : href;
                            htmlDoc = htmlWeb.Load(loaourl);
                            string gstr = htmlDoc.DocumentNode.InnerHtml;
                            list.Add(new Result
                            {
                                url = loaourl,
                                title = html.SelectSingleNode(xpath[searchText][2]).InnerText,
                                //date = DateTime.Parse(html.SelectSingleNode(xpath[searchText][3]).InnerText),
                                date = DateTime.Parse(DateSolve(html.SelectSingleNode(xpath[searchText][3]).InnerText.Replace('.', '-'))),
                                Where = searchText,
                                Type = searchText + "新闻",
                                // content = htmlDoc.DocumentNode.SelectSingleNode("//div[@class='Article_Content']/p[last()-1]").InnerHtml
                                //content = htmlDoc.DocumentNode.SelectSingleNode(xpath[searchText][4]).InnerHtml.Replace("src=\"", "src = \"" + SplitUrl(loaourl))
                                content = handleImage(xpath[searchText][4], loaourl)

                            });
                        }
                        catch (Exception)
                        {

                            //throw;
                            list.RemoveAt(list.Count - 1);
                        }
                    }
                );
            }
            ViewBag.Count = list.Count();
        }
        /// <summary>
        /// 将图片的地址改成可以访问的图片地址
        /// </summary>
        /// <param name="context"></param>
        /// <param name="link"></param>
        /// <returns></returns>
        public string handleImage(string context,string link) {

            string contextHtml = htmlDoc.DocumentNode.SelectSingleNode(context).InnerHtml;//.Replace("src='","src = '"+SplitUrl(link));
            if (contextHtml.Contains("src='".Trim()))
            {
                contextHtml = contextHtml.Replace("src='", "src = '" + SplitUrl(link));
            }
            else {
                contextHtml = contextHtml.Replace("src=\"", "src = \"" + SplitUrl(link));
            }
            return contextHtml;
        }
    }
}
